{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from brnn_model_2 import *\n",
    "import reader\n",
    "\n",
    "import subprocess\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "    Global variables\n",
    "\"\"\"\n",
    "model_type = \"test\"\n",
    "data_path = \"../data/\"\n",
    "save_path = \"./saved_model\"\n",
    "global_prior_pi = 0.25\n",
    "global_log_sigma1 = -1.0\n",
    "global_log_sigma2 = -7.0\n",
    "global_random_seed = 12\n",
    "global_num_gpus = 0\n",
    "\n",
    "\n",
    "# Model can be \"test\", \"small\", \"medium\", \"large\"\n",
    "model_select = \"test\"\n",
    "\n",
    "#Put the path to the data here\n",
    "dat_path = \"../data\"\n",
    "\n",
    "#Put the path to where you want to save the training data\n",
    "sav_path = \"tensorboard/\"\n",
    "\n",
    "# The mixing degree for the prior gaussian mixture\n",
    "# As in Fortunato they report scanning\n",
    "# mix_pi \\in { 1/4, 1/2, 3/4 }\n",
    "mixing_pi = 0.25\n",
    "\n",
    "# As in Fortunato they report scanning\n",
    "# log sigma1 \\in { 0, -1, -2 }\n",
    "# log sigma2 \\in { -6, -7, -8 }\n",
    "prior_log_sigma1 = -1.0\n",
    "prior_log_sigma2 = -7.0\n",
    "\n",
    "\n",
    "class SmallConfig(object):\n",
    "    \"\"\"Small config.\"\"\"\n",
    "    init_scale = 0.1\n",
    "    learning_rate = 1.0\n",
    "    max_grad_norm = 5\n",
    "    num_layers = 2\n",
    "    num_steps = 20\n",
    "    hidden_size = 200\n",
    "    max_epoch = 4\n",
    "    max_max_epoch = 13\n",
    "    keep_prob = 1.0\n",
    "    lr_decay = 0.5\n",
    "    \n",
    "    batch_size = 20\n",
    "    vocab_size = 10000\n",
    "    \n",
    "    X_dim = 200 # Size of the embedding\n",
    "\n",
    "class MediumConfig(object):\n",
    "    \"\"\"\n",
    "    Medium config.\n",
    "    Slightly modified according to email.\n",
    "    \"\"\"\n",
    "    init_scale = 0.05\n",
    "    learning_rate = 1.0\n",
    "    max_grad_norm = 5\n",
    "    num_layers = 2\n",
    "    num_steps = 35\n",
    "    hidden_size = 650\n",
    "    max_epoch = 20\n",
    "    max_max_epoch = 70\n",
    "    keep_prob = 1.0\n",
    "    lr_decay = 0.9\n",
    "    batch_size = 20\n",
    "    vocab_size = 10000\n",
    "\n",
    "    X_dim = 50 # Size of the embedding\n",
    "    \n",
    "class LargeConfig(object):\n",
    "    \"\"\"Large config.\"\"\"\n",
    "    init_scale = 0.04\n",
    "    learning_rate = 1.0\n",
    "    max_grad_norm = 10\n",
    "    num_layers = 2\n",
    "    num_steps = 35\n",
    "    hidden_size = 1500\n",
    "    max_epoch = 14\n",
    "    max_max_epoch = 55\n",
    "    keep_prob = 0.35\n",
    "    lr_decay = 1 / 1.15\n",
    "    batch_size = 20\n",
    "    vocab_size = 10000\n",
    "\n",
    "    X_dim = 100 # Size of the embedding\n",
    "    \n",
    "class TestConfig(object):\n",
    "    \"\"\"Tiny config, for testing.\"\"\"\n",
    "    init_scale = 0.1\n",
    "    learning_rate = 1.0\n",
    "    max_grad_norm = 1\n",
    "    num_layers = 2\n",
    "    num_steps = 20\n",
    "    hidden_size = 15\n",
    "    max_epoch = 1\n",
    "    max_max_epoch = 1\n",
    "    keep_prob = 1.0\n",
    "    lr_decay = 0.5\n",
    "    batch_size = 20\n",
    "    vocab_size = 10000\n",
    "\n",
    "    X_dim = 19 # Size of the embedding\n",
    "\n",
    "\n",
    "#    global_random_seed = set_random_seed\n",
    "    \n",
    "def get_config():\n",
    "    \"\"\"Get model config.\"\"\"\n",
    "    if model_type == \"small\":\n",
    "        config = SmallConfig()\n",
    "    elif model_type == \"medium\":\n",
    "        config = MediumConfig()\n",
    "    elif model_type == \"large\":\n",
    "        config = LargeConfig()\n",
    "    elif model_type == \"test\":\n",
    "        config = TestConfig()\n",
    "    else:\n",
    "        raise ValueError(\"Invalid model: %s\", model_type)\n",
    "\n",
    "    print (\"Model Type\")\n",
    "    print (model_type)\n",
    "    config.prior_pi = global_prior_pi\n",
    "    config.log_sigma1 = global_log_sigma1\n",
    "    config.log_sigma2 = global_log_sigma2\n",
    "\n",
    "    return config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test\n",
      "Model Type\n",
      "test\n",
      "Model Type\n",
      "test\n",
      "INFO:tensorflow:Summary name KL Loss is illegal; using KL_Loss instead.\n",
      "INFO:tensorflow:Summary name Total Loss is illegal; using Total_Loss instead.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "#    change_random_seed(global_random_seed)\n",
    "raw_data = reader.ptb_raw_data(data_path)\n",
    "train_data, valid_data, test_data, _ = raw_data\n",
    "\n",
    "print (model_type)\n",
    "\n",
    "config = get_config()\n",
    "eval_config = get_config()\n",
    "#eval_config.batch_size = 1\n",
    "#eval_config.num_steps = 1\n",
    "\n",
    "subprocess.Popen([\"tensorboard\",\"--logdir=tensorboard\"])\n",
    "\n",
    "with tf.Graph().as_default():\n",
    "    initializer = tf.random_uniform_initializer(-config.init_scale,\n",
    "                                                config.init_scale)\n",
    "\n",
    "    with tf.name_scope(\"Train\"):\n",
    "        train_input = PTBInput(config=config, data=train_data, name=\"TrainInput\")\n",
    "        with tf.variable_scope(\"Model\", reuse=None, initializer=initializer):\n",
    "            m = PTBModel(is_training=True, config=config, input_=train_input)\n",
    "        tf.summary.scalar(\"Training_Loss\", m.cost)\n",
    "        tf.summary.scalar(\"Learning_Rate\", m.lr)\n",
    "        tf.summary.scalar(\"KL Loss\", m.kl_loss)\n",
    "        tf.summary.scalar(\"Total Loss\", m.total_loss)\n",
    "\n",
    "    with tf.name_scope(\"Valid\"):\n",
    "        valid_input = PTBInput(config=config, data=valid_data, name=\"ValidInput\")\n",
    "        with tf.variable_scope(\"Model\", reuse=True, initializer=initializer):\n",
    "            mvalid = PTBModel(is_training=False, config=config, input_=valid_input)\n",
    "        tf.summary.scalar(\"Validation_Loss\", mvalid.cost)\n",
    "\n",
    "    with tf.name_scope(\"Test\"):\n",
    "        test_input = PTBInput(\n",
    "            config=eval_config, data=test_data, name=\"TestInput\")\n",
    "        with tf.variable_scope(\"Model\", reuse=True, initializer=initializer):\n",
    "            mtest = PTBModel(is_training=False, config=eval_config,\n",
    "                             input_=test_input)\n",
    "\n",
    "    models = {\"Train\": m, \"Valid\": mvalid, \"Test\": mtest}\n",
    "    for name, model in models.items():\n",
    "        model.export_ops(name)\n",
    "    metagraph = tf.train.export_meta_graph()\n",
    "    soft_placement = False\n",
    "    if global_num_gpus > 1:\n",
    "        soft_placement = True\n",
    "        util.auto_parallel(metagraph, m)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./saved_model/model.ckpt-0\n",
      "INFO:tensorflow:Starting standard services.\n",
      "INFO:tensorflow:Saving checkpoint to path ./saved_model/model.ckpt\n",
      "INFO:tensorflow:Starting queue runners.\n",
      "INFO:tensorflow:Model/global_step/sec: 0\n",
      "INFO:tensorflow:Recording summary at step 0.\n",
      "Epoch: 1 Learning rate: 1.000\n",
      "0.000 perplexity: 9869.366 speed: 670 wps\n",
      "KL is 9.601058006286621\n",
      "0.004 perplexity: 5544.689 speed: 2845 wps\n",
      "KL is 9.61371898651123\n"
     ]
    }
   ],
   "source": [
    "\n",
    "## Training !\n",
    "with tf.Graph().as_default():\n",
    "    tf.train.import_meta_graph(metagraph)\n",
    "    for model in models.values():\n",
    "        model.import_ops()\n",
    "    sv = tf.train.Supervisor(logdir=save_path)\n",
    "    config_proto = tf.ConfigProto(allow_soft_placement=soft_placement)\n",
    "    with sv.managed_session(config=config_proto) as session:\n",
    "\n",
    "        for i in range(config.max_max_epoch):\n",
    "            lr_decay = config.lr_decay ** max(i + 1 - config.max_epoch, 0.0)\n",
    "            m.assign_lr(session, config.learning_rate * lr_decay)\n",
    "\n",
    "            print(\"Epoch: %d Learning rate: %.3f\" % (i + 1, session.run(m.lr)))\n",
    "            train_perplexity = run_epoch(session, m, eval_op=m.train_op,\n",
    "                                         verbose=True)\n",
    "            print(\"Epoch: %d Train Perplexity: %.3f\" % (i + 1, train_perplexity))\n",
    "            valid_perplexity = run_epoch(session, mvalid)\n",
    "            print(\"Epoch: %d Valid Perplexity: %.3f\" % (i + 1, valid_perplexity))\n",
    "            \n",
    "        test_perplexity = run_epoch(session, mtest)\n",
    "        print(\"Test Perplexity: %.3f\" % test_perplexity)\n",
    "        \n",
    "        if save_path:\n",
    "            print(\"Saving model to %s.\" % save_path)\n",
    "            sv.saver.save(session, save_path, global_step=sv.global_step)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing\n",
      "INFO:tensorflow:Restoring parameters from ./saved_model/model.ckpt-0\n",
      "INFO:tensorflow:Starting standard services.\n",
      "INFO:tensorflow:Saving checkpoint to path ./saved_model/model.ckpt\n",
      "INFO:tensorflow:Starting queue runners.\n",
      "INFO:tensorflow:Error reported to Coordinator: <class 'ValueError'>, Fetch argument <tf.Tensor 'Test/Model/MultiRNNCellZeroState/BayesianLSTMCellZeroState/zeros:0' shape=(20, 15) dtype=float32> cannot be interpreted as a Tensor. (Tensor Tensor(\"Test/Model/MultiRNNCellZeroState/BayesianLSTMCellZeroState/zeros:0\", shape=(20, 15), dtype=float32) is not an element of this graph.)\n",
      "INFO:tensorflow:Model/global_step/sec: 0\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "Fetch argument <tf.Tensor 'Test/Model/MultiRNNCellZeroState/BayesianLSTMCellZeroState/zeros:0' shape=(20, 15) dtype=float32> cannot be interpreted as a Tensor. (Tensor Tensor(\"Test/Model/MultiRNNCellZeroState/BayesianLSTMCellZeroState/zeros:0\", shape=(20, 15), dtype=float32) is not an element of this graph.)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m/home/montoya/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, fetches, contraction_fn)\u001b[0m\n\u001b[1;32m    269\u001b[0m         self._unique_fetches.append(ops.get_default_graph().as_graph_element(\n\u001b[0;32m--> 270\u001b[0;31m             fetch, allow_tensor=True, allow_operation=True))\n\u001b[0m\u001b[1;32m    271\u001b[0m       \u001b[0;32mexcept\u001b[0m \u001b[0mTypeError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/montoya/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36mas_graph_element\u001b[0;34m(self, obj, allow_tensor, allow_operation)\u001b[0m\n\u001b[1;32m   2704\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_finalized\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2705\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_as_graph_element_locked\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mallow_tensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mallow_operation\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2706\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/montoya/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36m_as_graph_element_locked\u001b[0;34m(self, obj, allow_tensor, allow_operation)\u001b[0m\n\u001b[1;32m   2786\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgraph\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2787\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Tensor %s is not an element of this graph.\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2788\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: Tensor Tensor(\"Test/Model/MultiRNNCellZeroState/BayesianLSTMCellZeroState/zeros:0\", shape=(20, 15), dtype=float32) is not an element of this graph.",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-23-0b3c8cb088f8>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     12\u001b[0m        \u001b[0;31m# session = tf.Session()\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 14\u001b[0;31m         \u001b[0mtest_perplexity\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrun_epoch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmtest\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     15\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Test Perplexity: %.3f\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mtest_perplexity\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/montoya/Desktop/DeepLearning/deeplearningproject/test_area/brnn_model_2.py\u001b[0m in \u001b[0;36mrun_epoch\u001b[0;34m(session, model, eval_op, verbose)\u001b[0m\n\u001b[1;32m    373\u001b[0m     \u001b[0mcosts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0.0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    374\u001b[0m     \u001b[0miters\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 375\u001b[0;31m     \u001b[0mstate\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minitial_state\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    376\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    377\u001b[0m     fetches = {\n",
      "\u001b[0;32m/home/montoya/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    893\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    894\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 895\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    896\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    897\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/montoya/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1107\u001b[0m     \u001b[0;31m# Create a fetch handler to take care of the structure of fetches.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1108\u001b[0m     fetch_handler = _FetchHandler(\n\u001b[0;32m-> 1109\u001b[0;31m         self._graph, fetches, feed_dict_tensor, feed_handles=feed_handles)\n\u001b[0m\u001b[1;32m   1110\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1111\u001b[0m     \u001b[0;31m# Run request and get response.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/montoya/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, graph, fetches, feeds, feed_handles)\u001b[0m\n\u001b[1;32m    411\u001b[0m     \"\"\"\n\u001b[1;32m    412\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mgraph\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_default\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 413\u001b[0;31m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fetch_mapper\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_FetchMapper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfor_fetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    414\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fetches\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    415\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_targets\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/montoya/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mfor_fetch\u001b[0;34m(fetch)\u001b[0m\n\u001b[1;32m    231\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    232\u001b[0m       \u001b[0;31m# NOTE(touts): This is also the code path for namedtuples.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 233\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0m_ListFetchMapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    234\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    235\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0m_DictFetchMapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/montoya/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, fetches)\u001b[0m\n\u001b[1;32m    338\u001b[0m     \"\"\"\n\u001b[1;32m    339\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fetch_type\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 340\u001b[0;31m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_mappers\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0m_FetchMapper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfor_fetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mfetch\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    341\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_unique_fetches\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_value_indices\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_uniquify_fetches\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_mappers\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    342\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/montoya/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    338\u001b[0m     \"\"\"\n\u001b[1;32m    339\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fetch_type\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 340\u001b[0;31m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_mappers\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0m_FetchMapper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfor_fetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mfetch\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    341\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_unique_fetches\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_value_indices\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_uniquify_fetches\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_mappers\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    342\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/montoya/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mfor_fetch\u001b[0;34m(fetch)\u001b[0m\n\u001b[1;32m    231\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    232\u001b[0m       \u001b[0;31m# NOTE(touts): This is also the code path for namedtuples.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 233\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0m_ListFetchMapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    234\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    235\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0m_DictFetchMapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/montoya/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, fetches)\u001b[0m\n\u001b[1;32m    338\u001b[0m     \"\"\"\n\u001b[1;32m    339\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fetch_type\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 340\u001b[0;31m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_mappers\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0m_FetchMapper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfor_fetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mfetch\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    341\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_unique_fetches\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_value_indices\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_uniquify_fetches\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_mappers\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    342\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/montoya/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    338\u001b[0m     \"\"\"\n\u001b[1;32m    339\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fetch_type\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 340\u001b[0;31m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_mappers\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0m_FetchMapper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfor_fetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mfetch\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    341\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_unique_fetches\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_value_indices\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_uniquify_fetches\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_mappers\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    342\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/montoya/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mfor_fetch\u001b[0;34m(fetch)\u001b[0m\n\u001b[1;32m    239\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtensor_type\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    240\u001b[0m           \u001b[0mfetches\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcontraction_fn\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfetch_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 241\u001b[0;31m           \u001b[0;32mreturn\u001b[0m \u001b[0m_ElementFetchMapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetches\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcontraction_fn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    242\u001b[0m     \u001b[0;31m# Did not find anything.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    243\u001b[0m     raise TypeError('Fetch argument %r has invalid type %r' %\n",
      "\u001b[0;32m/home/montoya/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, fetches, contraction_fn)\u001b[0m\n\u001b[1;32m    275\u001b[0m       \u001b[0;32mexcept\u001b[0m \u001b[0mValueError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    276\u001b[0m         raise ValueError('Fetch argument %r cannot be interpreted as a '\n\u001b[0;32m--> 277\u001b[0;31m                          'Tensor. (%s)' % (fetch, str(e)))\n\u001b[0m\u001b[1;32m    278\u001b[0m       \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    279\u001b[0m         raise ValueError('Fetch argument %r cannot be interpreted as a '\n",
      "\u001b[0;31mValueError\u001b[0m: Fetch argument <tf.Tensor 'Test/Model/MultiRNNCellZeroState/BayesianLSTMCellZeroState/zeros:0' shape=(20, 15) dtype=float32> cannot be interpreted as a Tensor. (Tensor Tensor(\"Test/Model/MultiRNNCellZeroState/BayesianLSTMCellZeroState/zeros:0\", shape=(20, 15), dtype=float32) is not an element of this graph.)"
     ]
    }
   ],
   "source": [
    "## Testing\n",
    "print (\"Testing\")\n",
    "predicted = []   # Variable to store predictions\n",
    "with tf.Graph().as_default():\n",
    "    tf.train.import_meta_graph(metagraph)\n",
    "    for model in models.values():\n",
    "        model.import_ops()\n",
    "    sv = tf.train.Supervisor(logdir=save_path)\n",
    "    config_proto = tf.ConfigProto(allow_soft_placement=soft_placement)\n",
    "    with sv.managed_session(config=config_proto) as session:\n",
    "        \n",
    "       # session = tf.Session()\n",
    "    \n",
    "        test_perplexity = run_epoch(session, mtest)\n",
    "        print(\"Test Perplexity: %.3f\" % test_perplexity)\n",
    "\n",
    "        print (\"----------------------------------------------------------------\")\n",
    "        print (\"------------------ Prediction of Sentences ---------------------\")\n",
    "\n",
    "       #  inputs, predicted = fetch_output(session, mtest)\n",
    "\n",
    "        costs = 0.0\n",
    "        state = session.run(model.initial_state)\n",
    "\n",
    "        inputs = []\n",
    "        outputs = []\n",
    "        fetches = {\n",
    "            \"final_state\": model.final_state,\n",
    "            \"output\": model.output,\n",
    "            \"input\": model.input_data\n",
    "        }\n",
    "\n",
    "        for step in range(model.input.epoch_size):\n",
    "            feed_dict = {}\n",
    "            for i, (c, h) in enumerate(model.initial_state):\n",
    "                feed_dict[c] = state[i].c\n",
    "                feed_dict[h] = state[i].h\n",
    "\n",
    "            print (\"Computing batch %i/%i\"%(step, model.input.epoch_size))\n",
    "            vals = session.run(fetches, feed_dict)\n",
    "            state = vals[\"final_state\"]\n",
    "            output = vals[\"output\"]\n",
    "            input_i = vals[\"input\"]\n",
    "            outputs.append(output)\n",
    "            inputs.append(input_i)\n",
    "            \n",
    "            break;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input and output of the first chain of the first batch\n",
      "[ 102   14   24   32  752  381    2   29  120    0   35   92   60  111  143\n",
      "   32  616 3148  282   19]\n",
      "[[  9.79779288e-05   9.67627348e-05   1.01833539e-04 ...,   9.97266397e-05\n",
      "    1.02854683e-04   9.92195928e-05]\n",
      " [  9.79990655e-05   9.66500738e-05   1.01928919e-04 ...,   9.97894458e-05\n",
      "    1.02658414e-04   9.90654080e-05]\n",
      " [  9.80059049e-05   9.66006992e-05   1.01971855e-04 ...,   9.98253454e-05\n",
      "    1.02566286e-04   9.89926702e-05]\n",
      " ..., \n",
      " [  9.80024051e-05   9.65766812e-05   1.01997728e-04 ...,   9.99021358e-05\n",
      "    1.02503247e-04   9.89051550e-05]\n",
      " [  9.80098630e-05   9.65810832e-05   1.02007965e-04 ...,   9.99012118e-05\n",
      "    1.02495891e-04   9.88957108e-05]\n",
      " [  9.80073601e-05   9.65864310e-05   1.02010461e-04 ...,   9.99071417e-05\n",
      "    1.02503633e-04   9.88929241e-05]]\n"
     ]
    }
   ],
   "source": [
    "print (\"Input and output of the first chain of the first batch\")\n",
    "print (inputs[0][0])\n",
    "print (outputs[0][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(20, 10000)\n",
      "[8413 8413 8413 8413 8413 8413 8413 8413 8413 8413 8413 8413 8413 8413 8413\n",
      " 8413 8413 8413 8413 8413]\n"
     ]
    }
   ],
   "source": [
    "selected_words = np.argmax(outputs[0][0], axis = 1)\n",
    "print (outputs[0][0].shape)\n",
    "print (selected_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
